
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>Sample Benchmarks · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="../tutorials/" />
    
    
    <link rel="prev" href="./" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    What is ThingsJS?
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1.1" data-path="../about/Architecture.html">
            
                <a href="../about/Architecture.html">
            
                    
                    System Architecture
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.2" data-path="../about/Terminology.html">
            
                <a href="../about/Terminology.html">
            
                    
                    Terminology
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.3" data-path="../api/CLI.html">
            
                <a href="../api/CLI.html">
            
                    
                    CLI Commands
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.1.4" data-path="../Publications.html">
            
                <a href="../Publications.html">
            
                    
                    Publications
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="../GettingStarted.html">
            
                <a href="../GettingStarted.html">
            
                    
                    Getting Started
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="../tutorials/VirtualMachine.html">
            
                <a href="../tutorials/VirtualMachine.html">
            
                    
                    ThingsJS VM Image
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="../tutorials/Setup-PocketBeagle.html">
            
                <a href="../tutorials/Setup-PocketBeagle.html">
            
                    
                    Pocket Beagle
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="../tutorials/Setup-BeagleboneBlack.html">
            
                <a href="../tutorials/Setup-BeagleboneBlack.html">
            
                    
                    Beaglebone Black
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.4" data-path="./">
            
                <a href="./">
            
                    
                    Sample Components
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="1.2.5" data-path="RIoTBenchmarks.html">
            
                <a href="RIoTBenchmarks.html">
            
                    
                    Sample Benchmarks
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="../tutorials/">
            
                <a href="../tutorials/">
            
                    
                    Tutorials
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="../tutorials/FirstApplication.html">
            
                <a href="../tutorials/FirstApplication.html">
            
                    
                    Writing your First App
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="../tutorials/Migration.html">
            
                <a href="../tutorials/Migration.html">
            
                    
                    Live-Migration
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.3" data-path="../tutorials/dashboard/Dashboard.html">
            
                <a href="../tutorials/dashboard/Dashboard.html">
            
                    
                    Using the Dashboard
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="../api/">
            
                <a href="../api/">
            
                    
                    Developer API
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="../api/Devdocs.html">
            
                <a href="../api/Devdocs.html">
            
                    
                    Developer Documentation
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="../api/Testing.html">
            
                <a href="../api/Testing.html">
            
                    
                    Running Unit Tests
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="../api/Pubsub.html">
            
                <a href="../api/Pubsub.html">
            
                    
                    Pubsub
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.4" data-path="../api/Code.html">
            
                <a href="../api/Code.html">
            
                    
                    Code
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.5" data-path="../api/CodeEngine.html">
            
                <a href="../api/CodeEngine.html">
            
                    
                    CodeEngine
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.6" data-path="../api/Dispatcher.html">
            
                <a href="../api/Dispatcher.html">
            
                    
                    Dispatcher
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.7" data-path="../api/Scheduler.html">
            
                <a href="../api/Scheduler.html">
            
                    
                    Scheduler
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.8" data-path="../api/GFS.html">
            
                <a href="../api/GFS.html">
            
                    
                    File system
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.9" data-path="../api/Dashboard.html">
            
                <a href="../api/Dashboard.html">
            
                    
                    Web Dashboard
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.5" data-path="../Community/">
            
                <a href="../Community/">
            
                    
                    Community
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.5.1" data-path="../Community/involve.html">
            
                <a href="../Community/involve.html">
            
                    
                    Get involved
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.2" data-path="../Community/contribute.html">
            
                <a href="../Community/contribute.html">
            
                    
                    Contribute
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.3" data-path="../Community/feedback.html">
            
                <a href="../Community/feedback.html">
            
                    
                    Feedback
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.4" data-path="../Community/team.html">
            
                <a href="../Community/team.html">
            
                    
                    Team
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >Sample Benchmarks</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <h1 id="benchmark-apps">BENCHMARK Apps</h1>
<p>These benchmark applications are adapted from <a href="https://github.com/dream-lab/riot-bench" target="_blank">dreamlab RIOT benchmark</a>, and are intended to be used to test the different functionalities of the <code>ThingsJS</code> platform such as task scheduling. Everything can be found in <code>samples/IoTBench/.</code></p>
<h1 id="details">Details</h1>
<p>There are four benchmark applications: <code>ETL</code> (extract, transform, and load dataflow), <code>STATS</code> (statistical summarization dataflow), <code>TRAIN</code> (model training dataflow), and <code>PRED</code> (predictive analytics dataflow). The data comes from a .csv file containing 1000 lines of taxi data. For our implementation, each component along the dataflow communicates through <code>MQTT</code>. </p>
<h2 id="directory-structure">Directory structure</h2>
<pre><code>IotBench
&#x2502;   Spout.js - needed for all benchmarks
&#x2502;   Parse.js - needed for all benchmarks
&#x2502;
&#x2514;&#x2500;&#x2500;&#x2500;/ETL 
&#x2502;      |...all files for ETL benchmark
&#x2514;&#x2500;&#x2500;&#x2500;/PRED
&#x2502;      |...all files for PRED benchmark
&#x2514;&#x2500;&#x2500;&#x2500;/TRAIN
&#x2502;      |...all files for TRAIN benchmark
&#x2514;&#x2500;&#x2500;&#x2500;/STATS
&#x2502;      |...all files for STATS benchmark
&#x2514;&#x2500;&#x2500;&#x2500;/scripts - for benchmark measurements etc.
&#x2502;
&#x2514;&#x2500;&#x2500;&#x2500;/Data - all data files needed for benchmarks
|      | TAXI_properties.json -
|           a properties file for all the 
|           benchmarks. Please ensure 
|           paths pointed by this file exist 
|           locally (if you aren&apos;t using GFS) or
|           globally (if you are using GFS)
&#x2502;
&#x2514;&#x2500;&#x2500;&#x2500;/Models - all models needed for benchmarks
</code></pre><h3 id="etl">ETL:</h3>
<p>Filters and transforms sensor data before returning the data into SenML format once again.</p>
<p>Notes: </p>
<blockquote>
<ul>
<li>The interpolation component does no transformation for the taxi dataset (see <a href="https://arxiv.org/pdf/1701.08530.pdf#page=32" target="_blank">here</a>  for more information)</li>
<li>The external npm package <code>bloomfilter</code> is used in the bloom filter component</li>
<li>The Join component has been omitted</li>
<li>All components in the ported dataflow already communicate through MQTT, and therefore the MQTT Publish component is omitted due to redundancy</li>
</ul>
</blockquote>
<p> <img src="https://raw.githubusercontent.com/anshuiisc/FIG/master/ETL-1.png" alt="FCAST"></p>
<h3 id="stats">STATS:</h3>
<p>Performs numerous statistical summaries for a dataset and fits a simple linear regression model. </p>
<p>Notes:</p>
<blockquote>
<ul>
<li>The adapted GroupVIZ block creates a static webpage that displays a dynamic plot of the accumulated statistics</li>
</ul>
</blockquote>
<p><img src="https://raw.githubusercontent.com/anshuiisc/FIG/master/stats-1.png" alt="FCAST"></p>
<h3 id="train">TRAIN:</h3>
<p>Trains a regression tree and multi-variate model, with the option of saving these models to the global filesystem.</p>
<p>Notes:</p>
<blockquote>
<ul>
<li>The external <code>C4.5</code> npm package is used in place of Java&apos;s <code>J48</code> implementation of the C4.5 decision tree </li>
<li>The npm package <code>ml-regression-multivariate-linear</code> is used for training</li>
</ul>
</blockquote>
<p> <img src="https://raw.githubusercontent.com/anshuiisc/FIG/master/Train-1.png" alt="FCAST"></p>
<h3 id="pred">PRED:</h3>
<p>Fetches trained models saved in a filesystem (either locally or globally), and does predictions on the incoming data before calculating the residual error.</p>
<p>Notes:</p>
<blockquote>
<ul>
<li>On the taxi dataset, the prediction is on the taxi fee</li>
</ul>
</blockquote>
<p><img src="https://raw.githubusercontent.com/anshuiisc/FIG/master/pred-1.png" alt="FCAST"></p>
<h1 id="scheduling-benchmarks">Scheduling Benchmarks</h1>
<p>Instead of individually scheduling every component for each benchmark, we plan to use the <code>ThingsJS Scheduler</code> to dispatch these components automatically. The distribution of these tasks on workers can be based on each component&apos;s required memory, CPU, bandwidth, etc., decided by the scheduling algorithm used.</p>
<h1 id="user-guide">User Guide</h1>
<p>To run these benchmarks on a <code>ThingsJS worker</code>, users must have the appropriate dependencies. </p>
<ul>
<li>In the directory the worker is to run in, make sure it has been linked to <code>ThingsJS</code> using the command <code>npm link things-js</code>. Also ensure there is a <code>package.json</code> folder in the directory (this may just be an empty file)</li>
<li>If using the global filesystem, ensure that all datasets have been saved on the filesystem. Otherwise, make sure all datasets are saved in the worker directory</li>
</ul>
<p>A script is provided to automatically trigger the scheduling of a benchmark (provided there exists at least one running worker and a scheduler present). The command is:
<code>node benchmark.js &lt;ETL/STAT/TRAIN/PRED&gt;</code></p>
<h1 id="gathering-measurements">Gathering Measurements</h1>
<p>There are two scripts that currently gather data for these benchmarks, one for <strong>resource usage</strong>, and the other for <strong>line processing rate</strong>. These scripts can be found under <code>samples/IoTBench/scripts</code>.</p>
<h3 id="resource-usage">Resource Usage</h3>
<p>To gather information about the resource usage for each  component in the dataflow of a benchmark, run the command: 
<code>node resource_measurement.js &lt;cpu|memory|total&gt;</code>
Try to run this script as close as possible to when the benchmark is launched to avoid capturing irrelevant data.</p>
<ul>
<li>The data will be saved locally to a .csv file</li>
<li>You can either choose to capture CPU, memory, or both CPU and memory usage for the benchmark components </li>
</ul>
<h3 id="line-processing">Line Processing</h3>
<p>To gather information about how fast each line of data is being processed in the dataflow for any of the benchmarks, run the command: <code>node time_measurement.js</code> before launching a benchmark. This script will capture and accumulate the time measurements for how fast each component is processing a single line from the original source data.</p>
<p>To save this information locally, run the command
<code>node time_measurement.js &gt; filepath</code></p>
<p>The data captured by this script is in the following format:
<code>&lt;line number(s)&gt; &lt;component name&gt; &lt;processing time in ms&gt;</code> 
and also
<code>Total &lt;line number&gt; &lt;total time in ms&gt;</code> to indicate how long the line took to process through the entire dataflow.</p>
<p>All figures referenced from <a href="https://github.com/dream-lab/riot-bench" target="_blank">dreamlab RIOT benchmark</a></p>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="./" class="navigation navigation-prev " aria-label="Previous page: Sample Components">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
                <a href="../tutorials/" class="navigation navigation-next " aria-label="Next page: Tutorials">
                    <i class="fa fa-angle-right"></i>
                </a>
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"Sample Benchmarks","level":"1.2.5","depth":2,"next":{"title":"Tutorials","level":"1.3","depth":1,"path":"tutorials/README.md","ref":"tutorials/README.md","articles":[{"title":"Writing your First App","level":"1.3.1","depth":2,"path":"tutorials/FirstApplication.md","ref":"tutorials/FirstApplication.md","articles":[]},{"title":"Live-Migration","level":"1.3.2","depth":2,"path":"tutorials/Migration.md","ref":"tutorials/Migration.md","articles":[]},{"title":"Using the Dashboard","level":"1.3.3","depth":2,"path":"tutorials/dashboard/Dashboard.md","ref":"tutorials/dashboard/Dashboard.md","articles":[]}]},"previous":{"title":"Sample Components","level":"1.2.4","depth":2,"path":"samples/README.md","ref":"samples/README.md","articles":[]},"dir":"ltr"},"config":{"plugins":["jsdoc"],"root":"./guide","styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"jsdoc":{"options":"{}"},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"gitbook":"*"},"file":{"path":"samples/RIoTBenchmarks.md","mtime":"2018-12-17T22:41:57.665Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-04-24T23:23:12.622Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

